import requests
from bs4 import BeautifulSoup
headers = {
    'Cookie':'citydomain=bj; ganji_uuid=4833701062167349196422; _gl_tracker=%7B%22ca_source%22%3A%22-%22%2C%22ca_name%22%3A%22-%22%2C%22ca_kw%22%3A%22-%22%2C%22ca_id%22%3A%22-%22%2C%22ca_s%22%3A%22self%22%2C%22ca_n%22%3A%22-%22%2C%22ca_i%22%3A%22-%22%2C%22sid%22%3A51296451069%7D; __utma=32156897.1107777287.1525241697.1525241697.1525241697.1; __utmc=32156897; __utmz=32156897.1525241697.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmt=1; GANJISESSID=960n193u2r9jra75dof3k0c0fi; ganji_xuuid=141a791f-d43f-4c4a-9c4e-35217cce2ddc.1525241776606; xxzl_deviceid=qcrs8aaRoH%2F%2B3oG%2BqzHO%2F4Xd%2BEQqaAz8Zh8Atzis1yBNgOVW7hdtp9587DHV3A6H; 58uuid=ca04234d-1f02-47dd-954f-0832a3d10bc6; new_session=1; init_refer=; new_uv=1; ganji_login_act=1525241909664; __utmb=32156897.4.10.1525241697',
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36'
}
start_url = 'http://bj.ganji.com/wu'
data_url = 'http://bj.ganji.com'
def get_link(start_url):
    web_data = requests.get(start_url,headers=headers)
    soup = BeautifulSoup(web_data.text,'lxml')
    link = soup.select('dl.fenlei > dt > a')
    for i in link:
        update_url = data_url + str(i.get('href'))
        print(update_url)

get_link(start_url)



url_links = '''
http://bj.ganji.com/jiaju/
http://bj.ganji.com/rirongbaihuo/
http://bj.ganji.com/shouji/
http://bj.ganji.com/bangong/
http://bj.ganji.com/nongyongpin/
http://bj.ganji.com/jiadian/
http://bj.ganji.com/ershoubijibendiannao/
http://bj.ganji.com/ruanjiantushu/
http://bj.ganji.com/yingyouyunfu/
http://bj.ganji.com/diannao/
http://bj.ganji.com/xianzhilipin/
http://bj.ganji.com/fushixiaobaxuemao/
http://bj.ganji.com/meironghuazhuang/
http://bj.ganji.com/shuma/
http://bj.ganji.com/laonianyongpin/
http://bj.ganji.com/xuniwupin/
http://bj.ganji.com/qitawupin/
http://bj.ganji.com/ershoufree/
http://bj.ganji.com/wupinjiaohuan/

'''